import scrapy
from scrapy.utils.project import get_project_settings
from twisted.internet import reactor


class RankingListSpider(scrapy.Spider):
    name = 'ranking_list'
    allowed_domains = ['top.baidu.com']
    start_urls = ['https://top.baidu.com/board?tab=realtime']

    def parse(self, response, **kwargs):
        ranking_box = response.xpath('//main/div[2]/div/div[2]/div')
        for ranking in ranking_box:
            title = ranking.xpath('./div[2]/a/div[1]/text()').get()
            image = ranking.xpath('./a[1]/img/@src').get()
            hot_index = ranking.xpath('./div[1]/div[2]/text()').get()
            desc = ranking.xpath('./div[2]/div[1]/text()').get()
            print(title.strip(' '))
            print(image)
            print(hot_index.strip(' '))
            print(desc.strip(' '))


def run():
    from scrapy.crawler import CrawlerRunner

    # 使用项目配置
    runner = CrawlerRunner(get_project_settings())

    d = runner.crawl(RankingListSpider)

    # 添加爬虫执行结束的回调函数
    d.addBoth(lambda _: reactor.stop())

    # 开启事件循环
    reactor.run()


if __name__ == '__main__':
    run()